# coding=utf8

import re
import urllib2

from scrapy.selector import HtmlXPathSelector
from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor
from scrapy.contrib.spiders import CrawlSpider, Rule
from flash7k7k.items import FlashItem
from scrapy import log
from scrapy.stats import stats

stats.set_value('flash_7k7k_item', 0)
stats.set_value('flash_7k7k_item_ok', 0)
stats.set_value('flash_7k7k_swf', 0)
stats.set_value('flash_7k7k_htm', 0)
stats.set_value('flash_7k7k_htm_swf', 0)

def get_page(url):
    """
    return HTML or False on error
    """

    html = False

    try:
        req = urllib2.Request(url=url)
        f = urllib2.urlopen(req)

        html = unicode(f.read(), 'utf8', 'ignore')
    except urllib2.HTTPError:
        pass

    return html


categorys = {}

def get_category(cid):

    if categorys.get(cid):
        return categorys.get(cid)

    url = 'http://www.7k7k.com/flash_fl/%d_1.htm' % (cid,)

    # print ''
    # print url

    html = get_page(url);

    if not html:
        raise Exception('cannot get category page', url)

    try:
        match = re.search('_classname.*?"(.*?)"',html)
        category = match.group(1)
        # print flash
    except:
        raise Exception('cannot find category', url)

    categorys[cid] = category

    return category






class Flash7k7kSpider(CrawlSpider):
    name = 'flash7k7k'
    allowed_domains = ['7k7k.com']
    start_urls = ['http://www.7k7k.com/']

    rules = (
        Rule(SgmlLinkExtractor(allow=r'/flash/[0-9]+\.htm'), callback='parse_item', follow=True),
        Rule(SgmlLinkExtractor(allow=r'/flash_fl/.*\.htm'), follow=True),
        Rule(SgmlLinkExtractor(allow=r'/tag/'), follow=True),
    )

    def parse_item(self, response):

        stats.inc_value('flash_7k7k_item')

        hxs = HtmlXPathSelector(response)
        i = FlashItem()

        try:
            m = re.match(r'.*/flash/([0-9]+)\.htm', response.url)
            i['id'] = m.group(1)
        except:
            log.msg('cannot find id: %s' % (response.url,))
            return

        i['name'] = hxs.select('//div[@class="gate_c"]/div[@class="l"]/h1//text()').extract()[0]

        i['tag'] = ' '.join(hxs.select('//div[@class="gate_c"]/div[@class="r"]//li[@class="tag"]/a/text()').extract())

        try:
            match = re.search('_gameclass.*?([0-9]+).*?,',response.body)
            cid = match.group(1)
            cid = int(cid)
            # print cid
        except:
            raise Exception('cannot find category id', response.url)
        category = get_category(cid)
        # print category

        try:
            match = re.search('_gameroot.*?([0-9]+).*?,',response.body)
            cid2 = match.group(1)
            cid2 = int(cid2)
            # print cid2
        except:
            raise Exception('cannot find category id2', response.url)
        category2 = get_category(cid2)
        # print category2

        i['category'] = "%s %s" % (category2, category)


        r = hxs.select('//div[@class="gate_c"]/div[@class="r"]/p[@class="info"]//text()').extract()
        i['help'] = '\n'.join(r)

        try:
            match = re.search('_gamepath.*?"(.*?)"', response.body)
            i['flash'] = match.group(1)
            # print flash
        except:
            log.msg('cannot find flash %s' % (response.url,))

        try:
            match = re.search('_gamepic.*?"(.*?)"',response.body)
            i['pic_small'] = match.group(1)
            # print flash
        except:
            log.msg('cannot find small pic %s' % (response.url,))

        try:
            match = re.search('_gamebigpic.*?"(.*?)"',response.body)
            i['pic'] = match.group(1)
            # print flash
        except:
            log.msg('cannot find pic %s' % (response.url,))

        try:
            match = re.search('_gamepath.*?"(.*?)"',response.body)
            t_file = match.group(1)
            if t_file :
                if t_file[-4:] == '.swf':
                    stats.inc_value('flash_7k7k_swf')
                    i['flash'] = t_file
                else:
                    t_file = re.sub('\.htm$|\.html$','.swf',t_file)
                    if t_file[-4:] == '.swf' :
                        stats.inc_value('flash_7k7k_htm_swf')
                        i['flash'] = t_file
                        log.msg('ok, not direct flash %s, %s' % (response.url, t_file))
                    else:
                        stats.inc_value('flash_7k7k_htm')
                        log.msg('failed, not direct flash %s, %s' % (response.url, t_file))
            else:
                raise Exception('no file found')
        except:
            log.msg('cannot find flash %s' % (response.url,))
        

        
        # log.msg("%s %s" % (i['id'], i['flash']))

        stats.inc_value('flash_7k7k_item_ok')

        # print i
        return i


SPIDER = Flash7k7kSpider()
